using System;
using System.Text;
using System.Text.RegularExpressions;
using System.IO;
using System.Web;
using System.Diagnostics;

using MIL.Html;

namespace ComplianceFilter
{
	/// <summary>
	/// Reads HTML stream into a error correcting DOM and tinkers to force compliance with XHTML 1 transitional
	/// </summary>
	public class PageFilter : Stream
	{
		Stream          responseStream;
		long            position;
		StringBuilder   responseHtml;
		TraceSwitch		traceSwitch;
		bool			_ApplyMCMS;
		bool			_RemoveControlSpans;
        bool removeIDUnderScores;

		/// <summary>
		/// constructor
		/// </summary>
		/// <param name="inputStream"></param>
		public PageFilter (Stream inputStream)
		{
			responseStream = inputStream;
			responseHtml = new StringBuilder ();
			traceSwitch = new TraceSwitch("ComplianceFilter","Diagnostic output for trace filter");
			_ApplyMCMS = Configuration.ApplyMCMSRules;
			_RemoveControlSpans = Configuration.RemoveControlSpans;
            removeIDUnderScores = Configuration.RemoveIDUnderScores;
		}

		#region Filter overrides
		/// <summary>
		/// 
		/// </summary>
		public override bool CanRead
		{
			get
			{
				return true;
			}
		}
/// <summary>
/// 
/// </summary>
		public override bool CanSeek
		{
			get
			{
				return true;
			}
		}
/// <summary>
/// 
/// </summary>
		public override bool CanWrite
		{
			get
			{
				return true;
			}
		}
/// <summary>
/// 
/// </summary>
		public override void Close()
		{
			responseStream.Close ();
		}
/// <summary>
/// 
/// </summary>
		public override void Flush()
		{
			responseStream.Flush ();
		}
/// <summary>
/// 
/// </summary>
		public override long Length
		{
			get
			{
				return 0;
			}
		}

		/// <summary>
		/// 
		/// </summary>
		public override long Position
		{
			get
			{
				return position;
			}
			set
			{
				position = value;
			}
		}

		public override long Seek(long offset, SeekOrigin origin)
		{
			return responseStream.Seek (offset, origin);
		}

		public override void SetLength(long length)
		{
			responseStream.SetLength (length);
		}

		public override int Read(byte[] buffer, int offset, int count)
		{
			return responseStream.Read (buffer, offset, count);
		}
		#endregion

		#region Dirty work
		/// <summary>
		/// 
		/// </summary>
		/// <param name="buffer"></param>
		/// <param name="offset"></param>
		/// <param name="count"></param>
		public override void Write(byte[] buffer, int offset, int count)
		{
			Trace.WriteLineIf(true,"Filter buffering started at " + DateTime.Now.ToString());

			string strBuffer = System.Text.UTF8Encoding.UTF8.GetString (buffer, offset, count);

			// ---------------------------------
			// Wait for the closing </html> tag
			// ---------------------------------
			Regex eof = new Regex ("</html>", RegexOptions.IgnoreCase);

			if (!eof.IsMatch (strBuffer))
			{
				responseHtml.Append (strBuffer);
			}
			else
			{
				responseHtml.Append (strBuffer);
				string html = responseHtml.ToString();
				
				//this is to preserve the ie css comments tag if code
				string preserveIeCommentReg =@"<!--\[if.*\]>.*<!\[endif]-->";
				
				string Css = GetRegExMatchforCSS(html,preserveIeCommentReg);

				html = PreProcessHtml(html); //replace & with &amp; in content body
				HtmlDocument doc = HtmlDocument.Create(html,true);
				


				doc.DocTypeXHTML = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">";
			
				try
				{
					Trace.WriteLineIf(traceSwitch.TraceInfo,"DOM Processing started at" + DateTime.Now.ToString());

					//we have ie directive
					if(Css.Length!=0)
					{
						HtmlNodeCollection nodes =doc.Nodes.FindByName("link");
						if (nodes.Count!=0)
						{
							InsertIECss(nodes,Css);
						}
					}
					ProcessHtml(doc);


				}
				catch(Exception e)
				{
					Trace.WriteLineIf(traceSwitch.TraceError, "Filter failed" + e.Message + e.Source + e.StackTrace);
				}
				// Write the formatted HTML back
				string finalHtml="";
				if(!_ApplyMCMS)
				{
				 finalHtml=Finally(doc.XHTML);

				}
				else{
				 finalHtml= doc.XHTML;
				}

				finalHtml = FixAnyFunnies(finalHtml);
				byte[] data = System.Text.UTF8Encoding.UTF8.GetBytes (finalHtml);
	

				responseStream.Write (data, 0, data.Length);    
				
				Trace.WriteLineIf(traceSwitch.TraceInfo,"Filtering completed at " + DateTime.Now.ToString() + " " + data.Length + " bytes processed");

			}
		}

		/// <summary>
		/// somethings have escaped and got double escaped sort em out
		/// </summary>
		/// <param name="html"></param>
		/// <returns></returns>
		private string FixAnyFunnies(string html){
			html = html.Replace("&amp;nbsp","&nbsp;");
			html = html.Replace("</col>","");
			
			return html;

		}

		/// <summary>
		/// theres some funny busisness that needs fixing
		/// </summary>
		/// <param name="html"></param>
		/// <returns></returns>
		private string Finally(string html){
			html = html.Replace("td&gt;","");
			html = html.Replace("/--&gt;","");
			
			return html;
		}

		/// <summary>
		/// fudge to fix & site morse wants &amp;
		/// </summary>
		/// <param name="html"></param>
		/// <returns></returns>
		private string PreProcessHtml(string html){
			string reg = @"\\s&\\s";
			html = Regex.Replace(html,reg," &amp; ",RegexOptions.Multiline);
			return html;
		}


		/// <summary>
		/// used to remove dodgy chars from id and name in anchors
		/// </summary>
		/// <param name="doc"></param>
		private void ProcessAnchors(HtmlDocument doc){
			
			HtmlNodeCollection nodes = doc.Nodes.FindByName("a");
			//string reg="[^&a-zA-Z0-9_-]"; //get rid of any illegal char
			string temp="";
			foreach(HtmlElement elem in nodes)
			{
				
				if(elem.Attributes.IndexOf("id") != -1)
				{
					temp = elem.Attributes["id"].Value;
					temp = FixAttribute(temp,"_");
					elem.Attributes["id"].Value = temp;
				}
				if(elem.Attributes.IndexOf("name") != -1)
				{
					temp = elem.Attributes["name"].Value;
					temp = FixAttribute(temp,"_");
					elem.Attributes["name"].Value = temp;
				}
				if(elem.Attributes.IndexOf("title") != -1)
				{
					
					temp = elem.Attributes["title"].Value;
					//wnat iamge flow to work
                    //temp = FixAttribute(temp," ");
					elem.Attributes["title"].Value = temp;
				}
				
				
			}
		}

        /// <summary>
        /// 
        /// </summary>
        /// <param name="temp"></param>
        /// <param name="replacechar"></param>
        /// <returns></returns>
		private string FixAttribute(string temp, string replacechar){
			string reg="[^:&a-zA-Z0-9_-]"; //get rid of any illegal char
			temp = Regex.Replace(temp,reg,replacechar);
			if(temp.StartsWith(" ") ||temp.StartsWith("_") )
			{
				temp = temp.Remove(0,1);
			}
			//temp = temp.Replace("&","&amp;");
			return temp;
		}

		/// <summary>
		/// more replace & with &amp;
		/// </summary>
		/// <param name="doc"></param>
		private void ProcessMeta(HtmlDocument doc){
			HtmlNodeCollection nodes = doc.Nodes.FindByName("meta");
			
			foreach(HtmlElement elem in nodes)
			{
				if(elem.Attributes.IndexOf("content")!=-1){
					elem.Attributes["content"].Value = elem.Attributes["content"].Value.Replace(" & "," &amp; ");
				}
			}
		}

		/// <summary>
		/// using string manip and reg ex sticks in ie css tags
		/// </summary>
		/// <param name="nodes"></param>
		///<param name="cssHtml"></param>
		/// <returns></returns>
		private void InsertIECss(HtmlNodeCollection nodes,string cssHtml){
			
			//TODO JUST DO IT IN ONE
			
			HtmlText elem=new HtmlText(cssHtml);
		
			int i = nodes.Count;
			HtmlNode n = nodes[i-1];
			i = n.Index;
			n.Parent.Nodes.Insert(i+1,elem);
							
			Trace.WriteLineIf(traceSwitch.TraceInfo,"Inserted new link node for ie");
		
		}

		/// <summary>
		/// get rid of tbody may actually need tho
		/// </summary>
		/// <param name="doc"></param>
		private void RemoveTbody(HtmlDocument doc){
			HtmlNodeCollection nodes = doc.Nodes.FindByName("tbody");
			foreach(HtmlElement elem in nodes)
			{
				MoveContentOutofSpan(elem.Nodes,elem.Index+1);
				elem.Remove();
			}

		}

        /// <summary>
        /// get rid off anything cut and paste from word
        /// </summary>
        /// <param name="doc"></param>
        private void removeMSWordResidue(HtmlDocument doc)
        {
            HtmlNodeCollection nodes = doc.Nodes.FindByName("p");
            foreach (HtmlElement elem in nodes)
            {
                if (elem.Nodes.Count == 0 && elem.Text.Length == 0)
                {
                    //kill empty p tags
                    elem.Remove();
                }
                else{
                    if (elem.HTML == "<p>&nbsp;</p>")
                    {
                        elem.Remove();
                    }
                    else
                    {
                        if (elem.Attributes.IndexOf("class") != -1 && elem.Attributes["class"].Value == "MsoNormal")
                        {
                            elem.Attributes.RemoveAt(elem.Attributes.IndexOf("class"));
                        }
                        if (elem.Attributes.IndexOf("style") != -1 )
                        {
                            elem.Attributes.RemoveAt(elem.Attributes.IndexOf("style"));
                        }
                    }
                }
                

            }

            //nodes = doc.Nodes.FindByName("span");
            //foreach (HtmlElement elem in nodes)
            //{
            //    if (elem.Nodes.Count == 0 && elem.Text.Length == 0)
            //    {
            //        elem.Remove();
            //    }
            //    else if (elem.HTML == "<span>&nbsp; </span>" || elem.HTML == "<span> </span>")
            //    {
            //        elem.Remove();
            //    }
            //}
        }

		/// <summary>
		/// 
		/// </summary>
		/// <param name="doc"></param>
		private void RemoveDeprecated(HtmlDocument doc)
		{
			//Remove onfoucusout event handlers
			HtmlNodeCollection nodes = doc.Nodes.FindByAttributeName("onfocusout");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("onfocusout") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"onfocusout attribute removed from:" + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("onfocusout"));
					}
				}
			}

			//Kill bgcolor
			nodes = doc.Nodes.FindByAttributeName("bgcolor");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("bgcolor") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"bgcolor attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("bgcolor"));
					}
				}
			}


			//Kill target="_blank"
//			nodes = doc.Nodes.FindByAttributeName("target");
//			if(nodes.Count != 0)
//			{
//				foreach(HtmlElement elem in nodes)
//				{
//					if(elem.Attributes.IndexOf("target") != -1)
//					{
//						Trace.WriteLineIf(traceSwitch.TraceInfo,"target attribute removed from: " + elem.XHTML);
//
//						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("target"));
//					}
//				}
//			}



			//Remove id from title tag
			nodes = doc.Nodes.FindByName("title");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("id") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"id attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("id"));		
					}
					if(elem.Attributes.IndexOf("title") != -1)
					{
						elem.Attributes["title"].Value = elem.Attributes["title"].Value.Replace(",","").Replace("'","");
					}
					
				}
			}

			//Remove width attributes
			nodes = doc.Nodes.FindByAttributeName("width");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Name != "table" && elem.Name != "img" && elem.Name != "iframe"  && elem.Attributes.IndexOf("width") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"width attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("width"));
					}
				}
			}

			//Remove height attributes
			nodes = doc.Nodes.FindByAttributeName("height");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					//removed elem.Name != "table" && 
					if(elem.Name != "img" && elem.Name != "iframe" && elem.Attributes.IndexOf("height") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"height attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("height"));
					}
				}
			}


			//Remove hspace attributes
			nodes = doc.Nodes.FindByAttributeName("hspace");
			if(nodes.Count != 0)			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("hspace") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"hspace attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("hspace"));
					}
				}
			}

			//Remove align attributes
			nodes = doc.Nodes.FindByAttributeName("align");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("align") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"align attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("align"));
					}
				}
			}


			//Remove type attributes from selects
			nodes = doc.Nodes.FindByAttributeName("type");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Name == "select" || elem.Name=="ul")
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"type attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("type"));
					}
				}
			}

			//Remove name tags from forms, they are invalid
			nodes = doc.Nodes.FindByName("form");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("name") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"name attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("name"));	
					}
					if(elem.Attributes.IndexOf("language") != -1)
					{
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("language"));	
					}

				}
			}

			//Translate u elements into spans
			nodes = doc.Nodes.FindByName("u");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					elem.Name = "span";
				}
			}

			

			//Remove border tags from images, transpose longDesc to alt
			nodes = doc.Nodes.FindByName("img");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
                    bool isGoogleImage = isGoogleImg(elem);
                    if(elem.Attributes.IndexOf("border") != -1 && !isGoogleImage)
					{	
						Trace.WriteLineIf(traceSwitch.TraceInfo,"border attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("border"));
					}
					if(elem.Attributes.IndexOf("vspace") != -1)
					{	
						Trace.WriteLineIf(traceSwitch.TraceInfo,"vspace attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("vspace"));
					}
					if(elem.Attributes.IndexOf("hspace") != -1)
					{	
						Trace.WriteLineIf(traceSwitch.TraceInfo,"hspace attribute removed from: " + elem.XHTML);

						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("hspace"));
					}
					if(elem.Attributes.IndexOf("longdesc") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"longdesc attribute substituted for alt at: " + elem.XHTML);

						if(elem.Attributes.IndexOf("alt") == -1)
						{
							elem.Attributes.Add(new HtmlAttribute("alt",elem.Attributes["longDesc"].Value));
						}
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("longdesc"));
					}
					//added this to get rid of telerik gubbins
					if(elem.Attributes.IndexOf("editorid") != -1)
					{
						Trace.WriteLineIf(traceSwitch.TraceInfo,"removed editorid: " + elem.XHTML);
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("editorid"));
					}

					if(elem.Attributes.IndexOf("alt") != -1)
					{
						string temp = elem.Attributes["alt"].Value;
						//want encoded html in attribute to allow image flow to work
                        //temp  = FixAttribute(temp," ");
						elem.Attributes["alt"].Value = temp;
					}
					
					else{
						//no alt put a blank one in 
                        if (!isGoogleImage)
                        {
                            elem.Attributes.Add(new HtmlAttribute("alt", " "));
                        }
					}
                    //if parent is p then move it out of p and remove the p
                    //if (elem.Parent.Name == "p")
                    //{
                    //    int i = elem.Parent.Index;
                    //    HtmlElement parent = elem.Parent.Parent;
                    //    HtmlElement img = elem;
                    //    elem.Remove();
                    //    parent.Nodes.Insert(i, img);
                    //}
				}
			}
              //added this flag becuase asp.net20 / ajax seems to die when we apply this fix which is sitemorse specific
              //if (removeIDUnderScores){
			   //remove .net clientside validation stuff on spans
               //nodes=doc.Nodes.FindByName("span");
               // if (nodes.Count != 0)
               // {
               //     foreach (HtmlElement elem in nodes)
               //     {

               //         if (elem.Attributes.IndexOf("controltovalidate") != -1)
               //         {
               //             elem.Attributes.RemoveAt(elem.Attributes.IndexOf("controltovalidate"));
               //         }
               //         if (elem.Attributes.IndexOf("errormessage") != -1)
               //         {
               //             elem.Attributes.RemoveAt(elem.Attributes.IndexOf("errormessage"));
               //         }
               //         if (elem.Attributes.IndexOf("evaluationfunction") != -1)
               //         {
               //             elem.Attributes.RemoveAt(elem.Attributes.IndexOf("evaluationfunction"));
               //         }
               //         if (elem.Attributes.IndexOf("clientvalidationfunction") != -1)
               //         {
               //             elem.Attributes.RemoveAt(elem.Attributes.IndexOf("clientvalidationfunction"));
               //         }

               //     }
               //}
			//}


            ////get rid of mcms radedit bits
            //if(_ApplyMCMS)
            //{
            //    if(_RemoveControlSpans)
            //    {
            //        nodes=doc.Nodes.FindByName("span");
            //        if(nodes.Count!=0)
            //        {
            //            foreach(HtmlElement elem in nodes)
            //            {
            //                //does the span contain p if so then its not valid
            //                //also if its image placeholder sometimes more than one present with same id this causes failure
            //                //may need to put in something extra for xml placeholder?
            //                //for site map which is used using control and ul li we need to get rid of span
            //                //may also need test for h2 
            //                if(ContainsHtml(elem.HTML) || elem.HTML.IndexOf("SingleImagePlaceholderControl")!=-1 || elem.HTML.IndexOf("id=\"Sitemap")!=-1)
            //                {									
								
            //                    MoveContentOutofSpan(elem.Nodes,elem.Index+1);
            //                    elem.Remove();
            //                    Trace.WriteLineIf(traceSwitch.TraceInfo,"removed span for mcms: ");

            //                }
            //                else if(elem.Attributes.IndexOf("editcontrolheight")!=-1 || elem.Attributes.IndexOf("editcontrolwidth")!=-1){ //its empty tag but placeholder
            //                    elem.Remove();
            //                }

            //            }
            //        }
            //    }
            //}

		}

        /// <summary>
        /// if we have google image dont close it
        /// </summary>
        /// <param name="elem"></param>
        /// <returns></returns>
        private bool isGoogleImg(HtmlElement elem)
        {
            if (elem.Attributes.IndexOf("src") != -1 && elem.Attributes["src"].Value.Contains("www.googleadservices.com/pagead/conversion")) {
                elem.IsTerminated = false;
                elem.IsExplicitlyTerminated = false;
                elem.IsGoogleImage = true;
                return true;
            }
            return false;
        }

		/// <summary>
		/// does the node contain any html
		/// </summary>
		/// <param name="html"></param>
		/// <returns></returns>
		private bool ContainsHtml(string html){
			string reg="<.*>";
			if(GetRegExMatch(html,reg).Length!=0)
			{
				return true;
			}
			else{
				return false;
			}
		}

		/// <summary>
		/// 
		/// </summary>
		/// <param name="nodes"></param>
		/// <param name="spanIndex"></param>
		private void MoveContentOutofSpan(HtmlNodeCollection nodes,int spanIndex){
			//Move table elements outside of spans
			int i=0;
			if(nodes.Count != 0)
			{

				foreach(HtmlNode node in nodes){
					try{		
						node.Parent.Parent.Nodes.Insert(spanIndex+i,node);					
						i++;
					}
					catch(Exception e)
					{
						Trace.WriteLineIf(traceSwitch.TraceError,e.Message.ToString());
					}
				}
	
			}
		}

		/// <summary>
		/// 
		/// </summary>
		/// <param name="doc"></param>
		private void ProcessIDs(HtmlDocument doc)
		{
			//Wrap __VIEWSTATE,__EVENTTARGET and __EVENTARGS in a div
			HtmlNodeCollection nodes = doc.Nodes.FindByName("input");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					//get rid of border attribute
					if(elem.Attributes.IndexOf("border")!=-1){
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("border"));
					}

					if(elem.Attributes.IndexOf("name") != -1)
					{
						HtmlElement containerDiv = null;
						if(elem.Attributes["name"].Value == "__EVENTTARGET"   || 
							elem.Attributes["name"].Value == "__EVENTARGUMENT" ||
							elem.Attributes["name"].Value == "__VIEWSTATE"	)
						{
							if(containerDiv == null) 
							{
								containerDiv = new HtmlElement("div");
								elem.Parent.Nodes.Insert(0,containerDiv);
							}

							elem.Remove();
							containerDiv.Nodes.Add(elem);
							Trace.WriteLineIf(traceSwitch.TraceInfo,"ASP .net form variable placed in div element: " + elem.XHTML);

						}
					}					
				}
			}

			
			//Fix script specification
			nodes = doc.Nodes.FindByAttributeName("language");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes.IndexOf("type") != -1)
					{
						//Remove language
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("language"));
						elem.Attributes.Add(new HtmlAttribute("type","text/javascript"));
						Trace.WriteLineIf(traceSwitch.TraceInfo,"language attribute removed from: " + elem.XHTML);
					}
					else
					{
						//Replace language 
						elem.Attributes.RemoveAt(elem.Attributes.IndexOf("language"));
						if(elem.Name!="select")
						{
							elem.Attributes.Add(new HtmlAttribute("type","text/javascript"));
						}
					
						Trace.WriteLineIf(traceSwitch.TraceInfo,"language attribute substituted for type at: " + elem.XHTML);
					}
				}
			}

            
			//Remove leading underscores from all id's
			nodes = doc.Nodes.FindByAttributeName("id");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes["id"].Value.StartsWith("__"))				
					{
						elem.Attributes["id"].Value = elem.Attributes["id"].Value.Remove(0,2);
						Trace.WriteLineIf(traceSwitch.TraceInfo,"Leading '__' attribute removed from id attribute at: " + elem.XHTML);
					}
					else if(elem.Attributes["id"].Value.StartsWith("_"))
					{
						elem.Attributes["id"].Value = elem.Attributes["id"].Value.Remove(0,1);						
						Trace.WriteLineIf(traceSwitch.TraceInfo,"Leading '_' attribute removed from id attribute at: " + elem.XHTML);
					}
		
					if(_ApplyMCMS){
						elem.Attributes["id"].Value = elem.Attributes["id"].Value.Replace("'","").Replace(",","");
					}
				}
			}

			//Remove leading underscores from all for
			nodes = doc.Nodes.FindByAttributeName("for");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
					if(elem.Attributes["for"].Value.StartsWith("__"))				
					{
						elem.Attributes["for"].Value = elem.Attributes["for"].Value.Remove(0,2);
						Trace.WriteLineIf(traceSwitch.TraceInfo,"Leading '_' attribute removed from for attribute at: " + elem.XHTML);

					}
					else if(elem.Attributes["for"].Value.StartsWith("_"))
					{
						elem.Attributes["for"].Value = elem.Attributes["for"].Value.Remove(0,1);						
						Trace.WriteLineIf(traceSwitch.TraceInfo,"Leading '_' attribute removed from for attribute at: " + elem.XHTML);
	
					}		
				}
			}


			//Replace __doPostBack's implementation and update id references in scripts
			nodes = doc.Nodes.FindByName("script");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement elem in nodes)
				{
                    //we dont want script tags ending like <script />
                    elem.IsExplicitlyTerminated = true;
                    try
					{				
						HtmlText textNode = elem.Nodes[0] as HtmlText;

						Trace.WriteLineIf(traceSwitch.TraceInfo, "Javascript processed (original): " + textNode.Text);
	
						if(textNode.Text.IndexOf("function __doPostBack") != -1)
						{
							int     pos1 = elem.Text.IndexOf ("var theform;");
							int     pos2 = elem.Text.IndexOf ("theform.__EVENTTARGET", pos1);
							string  methodText = elem.Text.Substring (pos1, pos2-pos1);
						
							string  formID = ((HtmlElement)	doc.Nodes.FindByName("form")[0]).Attributes["id"].Value;
						
						
							textNode.Text = textNode.Text.Replace (methodText,  
								@"var theform = document.getElementById ('" + formID + "');\r\n\t\t");

								textNode.Text = textNode.Text.Replace("theform.__EVENTTARGET.value = eventTarget.split(\"$\").join(\":\");", 
									"theform.__EVENTTARGET.value = eventTarget.split(\"$\").join(\":\");");

						}

						else
						{
							textNode.Text = textNode.Text.Replace("(\"_","(\"");
						}

						Trace.WriteLineIf(traceSwitch.TraceInfo, "Javascript processed (altered): " + textNode.Text);
						
					}
					catch
					{
						Trace.WriteLineIf(traceSwitch.TraceError,"Error processing javascript");
					}
				}
			}

			//Encode hrefs
			nodes = doc.Nodes.FindByAttributeName("href");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{

					if(node.Attributes["href"].Value.ToLower().IndexOf("javascript:") == -1)
					{
						
					}
					else
					{
						//Fix id in href
						node.Attributes["href"].Value = node.Attributes["href"].Value.Replace("'_","'");
						Trace.WriteLineIf(traceSwitch.TraceInfo,"Leading '_' removed from href attribute value at: " + node.XHTML);
	
					}
					if(node.Attributes["href"].Value.IndexOf("ILINK|") == -1)
					{
						node.Attributes["href"].Value = HtmlAttribute.SafeAttributeEncode(node.Attributes["href"].Value);
					}
					if(node.Attributes.IndexOf("id")!=-1){
						node.Attributes["id"].Value = node.Attributes["id"].Value.Replace(" ","_");
					}
					if(node.Attributes.IndexOf("name")!=-1)
					{
						node.Attributes["name"].Value = node.Attributes["name"].Value.Replace(" ","_");
					}
				}
			}
			//Encode action
			nodes = doc.Nodes.FindByAttributeName("action");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					if(node.Attributes["action"].Value.IndexOf("ILINK|") == -1)
					{
						node.Attributes["action"].Value = HtmlAttribute.SafeAttributeEncode(node.Attributes["action"].Value);
					}

				}
			}

			//fix zref 
			if(_ApplyMCMS){
				nodes = doc.Nodes.FindByAttributeName("zref");
				if(nodes.Count!=0){
					foreach(HtmlElement node in nodes)
					{
						node.Remove();
					}
				}
			}

		}

		/// <summary>
		/// used to get content using reg ex
		/// </summary>
		/// <param name="strSearchString"></param>
		/// <param name="strSearchPattern"></param>
		/// <returns></returns>
		private string GetRegExMatch(string strSearchString,string strSearchPattern)
		{
			 
			Regex objReg = new Regex(strSearchPattern,RegexOptions.IgnoreCase); 			
			return objReg.Match(strSearchString).Value; 

		}

		/// <summary>
		/// 
		/// </summary>
		/// <param name="strSearchString"></param>
		/// <param name="strSearchPattern"></param>
		/// <returns></returns>
		private string GetRegExMatchforCSS(string strSearchString,string strSearchPattern)
		{
			 
			Regex objReg = new Regex(strSearchPattern,RegexOptions.Singleline); 			
			return objReg.Match(strSearchString).Value; 

		}	


		/// <summary>
		/// 
		/// </summary>
		/// <param name="doc"></param>
		private void ProcessHtml(HtmlDocument doc)
		{
			RemoveDeprecated(doc);
			
			ProcessIDs(doc);
			
			ProcessAnchors(doc);
			
			//RemoveTbody(doc);

			processULs(doc);

            removeMSWordResidue(doc);
			
			ProcessMeta(doc);

			//Kill xml namespace spam
			HtmlNodeCollection nodes = doc.Nodes.FindByName("?xml:namespace");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					Trace.WriteLineIf(traceSwitch.TraceInfo,"xml namespace declaration removed " + node.XHTML);
	
					node.Remove();
				}
			}

			//Kill valign=center
			nodes = doc.Nodes.FindByName("td");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					Trace.WriteLineIf(traceSwitch.TraceInfo,"td cleaned" + node.XHTML);
	
					if(node.Attributes["valign"] != null && node.Attributes["valign"].Value.ToLower() == "center")
					{
						node.Attributes.RemoveAt(node.Attributes.IndexOf("valign"));
					}
					else if(node.Attributes["valign"] != null && node.Attributes["valign"].Value.ToLower() == "top"){
						//lcase Top 
						node.Attributes["valign"].Value = "top";
					}
						
					//breaks accessibility
					if(node.Attributes.IndexOf("background")!=-1){
						node.Attributes.RemoveAt(node.Attributes.IndexOf("background"));
					}
				}
			}

			//Kill center tags, replace with span
			nodes = doc.Nodes.FindByName("center");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					node.Name = "span";
					Trace.WriteLineIf(traceSwitch.TraceInfo,"<center> replaced " + node.XHTML);
	
					node.Attributes.Clear();
				}
			}
			
			//Kill nowrap tags, replace with span
			nodes = doc.Nodes.FindByName("nowrap");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					node.Name = "span";
					Trace.WriteLineIf(traceSwitch.TraceInfo,"<center> replaced " + node.XHTML);
	
					node.Attributes.Clear();
				}
			}
			
			//Kill font tags , replace with span
			nodes = doc.Nodes.FindByName("font");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					node.Name = "span";
					Trace.WriteLineIf(traceSwitch.TraceInfo,"<center> replacs " + node.XHTML);
	
					node.Attributes.Clear();
				}
			}

			//Force explicit termination of div's and anchors <div></div> <a></a> 
			nodes = doc.Nodes.FindByName("div");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					node.IsExplicitlyTerminated = true;
					//is parent node a p tag if it is then move and get rid					
					if(node.Parent.Name=="p"){
						int i = node.Parent.Index;
						HtmlElement parent = node.Parent.Parent;
						HtmlElement div = node;						
						node.Remove();
						parent.Nodes.Insert(i,div);
					}
				}
			}


			//Move table elements outside of spans
			nodes = doc.Nodes.FindByName("table");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					try
					{
						if(node.Parent != null && node.Parent.Name == "span")
						{
							int spanIndex = node.Parent.Index;
							node.Remove();
							node.Parent.Parent.Nodes.Insert(spanIndex,node);

						}
					}
					catch(Exception e)
					{e.Message.ToString();}
				}
			}

			AddMeta(doc,"RiverInteractive.ComplianceFilterTest",System.Reflection.Assembly.GetAssembly(this.GetType()).ImageRuntimeVersion.ToString());
		}



		/// <summary>
		/// due to xslt in umbraco we sometimes end up with blank uls ie <ul></ul>
		/// </summary>
		/// <param name="doc"></param>
		private void processULs(HtmlDocument doc){
			HtmlNodeCollection nodes = doc.Nodes.FindByName("ul");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{									
					//HttpContext.Current.Response.Write("<!--ul code is " + node.HTML + " child count is " + node.Nodes.Count + "-->");
                    if (node.Nodes.Count == 1 || node.Nodes.Count == 0)
                    {
						node.Remove();
					}					
				}
			}
		}

		/// <summary>
		/// 
		/// </summary>
		/// <param name="doc"></param>
		/// <param name="name"></param>
		/// <param name="content"></param>
		private void AddMeta(HtmlDocument doc,string name,string content)
		{
			//Add meta tag
			HtmlNodeCollection nodes = doc.Nodes.FindByName("title");
			if(nodes.Count != 0)
			{
				foreach(HtmlElement node in nodes)
				{
					HtmlElement meta = new HtmlElement("meta");
					meta.Attributes.Add(new HtmlAttribute("name",name));
					meta.Attributes.Add(new HtmlAttribute("content",content));
					node.Parent.Nodes.Insert(node.Index + 1,meta);	
				}
			}
		}

		#endregion

	}
}